package co.edu.unal.bioingenium.kbmed.experimentation;

import co.edu.unal.bioingenium.kbmed.config.Configuration;
import co.edu.unal.bioingenium.kbmed.knowledge.KnowledgeSourceMetaData;
import co.edu.unal.bioingenium.kbmed.knowledge.dao.KnowledgeSourceDAO;
import co.edu.unal.bioingenium.kbmed.knowledge.dao.KnowledgeSourceDAOFactory;
import co.edu.unal.bioingenium.kbmed.knowledge.snomed.SnomedMetaData;
import co.edu.unal.bioingenium.kbmed.retrieval.text.process.PrepareByWords;
import co.edu.unal.bioingenium.kbmed.text.filter.FilterPipe;
import co.edu.unal.bioingenium.kbmed.text.representation.io.CorpusIO;
import co.edu.unal.bioingenium.kbmed.text.representation.vo.Corpus;
import co.edu.unal.bioingenium.kbmed.util.io.FileManager;
import co.edu.unal.bioingenium.kbmed.util.time.TimeUtil;
import gnu.trove.map.TObjectIntMap;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
 *
 * @author Luis A Riveros
 */
public class ConceptWeirdness {

    private Map<String, Double> generalDictionarty;
    private Map<String, Double> specializedDictionarty;
    private int totalRAE = 152558294;
    private String RAE_DICTIONARY_PATH;
    private final int dictionaryWords = 20000;
    private final String startDate;

    public ConceptWeirdness(String RAE_DICTIONARY_PATH) {
        this.RAE_DICTIONARY_PATH = RAE_DICTIONARY_PATH;
        generalDictionarty = new HashMap<String, Double>(dictionaryWords);
        specializedDictionarty = new HashMap<String, Double>(dictionaryWords);
        startDate = TimeUtil.getCurrentDate();
    }

    public void conceptAllTermsCorpus() {
        KnowledgeSourceDAO knowledgeSourceDAO = KnowledgeSourceDAOFactory.getKnowledgeSourceDAO();
        PrepareByWords prepareCollection = new PrepareByWords();
        Corpus corpus;
        Map<String, Set<String>> termsByconceptIds = knowledgeSourceDAO.getTermsByConceptId();
        Map<String, String> corpusData = new HashMap<String, String>(termsByconceptIds.size());
        StringBuilder stringBuilder = new StringBuilder();
        for (String conceptId : termsByconceptIds.keySet()) {
            stringBuilder.setLength(0);
            for (String term : termsByconceptIds.get(conceptId)) {
                stringBuilder.append(term).append(" ");
            }
            corpusData.put(conceptId, stringBuilder.toString());
        }
        corpus = prepareCollection.buildCorpus("conceptAllTerms", corpusData, 0, 0);
        CorpusIO.storeCorpus(corpus, "conceptAllTerms.dat");
    }

    public void filterRAEDictionary() throws IOException {
        BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(new FileInputStream(RAE_DICTIONARY_PATH), "UTF8"));
        String line;
        StringTokenizer stringTokenizer;
        bufferedReader.readLine();
        StringBuilder sb = new StringBuilder();
        String word, freq, nfreq;
        BufferedWriter bufferedWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(Configuration.EXPERIMENTS_DIR + File.separator + "RAE-" + dictionaryWords + "words.txt")));
        int count = 0;
        while ((line = bufferedReader.readLine()) != null) {
            sb.setLength(0);
            stringTokenizer = new StringTokenizer(line);
            stringTokenizer.nextToken();
            word = stringTokenizer.nextToken();
            freq = stringTokenizer.nextToken();
            for (int i = 0; i < freq.length(); i++) {
                if (freq.charAt(i) != ',') {
                    sb.append(freq.charAt(i));
                }
            }
            freq = sb.toString();
            nfreq = stringTokenizer.nextToken();
            count += Integer.parseInt(freq);
            bufferedWriter.write(word + "," + freq + "," + nfreq + "," + ((double) Integer.parseInt(freq) / (double) totalRAE) + "\n");
        }
        bufferedReader.close();
        bufferedWriter.flush();
        bufferedWriter.close();

        System.out.println("Total: " + count);
    }

    public void loadDictionaries() {
        FilterPipe filterPipe = new FilterPipe();
        filterPipe.setMinTokenLength(1);
        filterPipe.setMaxTokenLength(20);
        filterPipe.setRemoveStopWords(true);
        filterPipe.setRemoveNumbers(true);
        String word2;
        try {
            //General Dictionary
            String savePath = Configuration.EXPERIMENTS_DIR + File.separator + ConceptWeirdness.class.getSimpleName();
            FileManager.verifyAndCreatePath(savePath);
            BufferedReader bufferedReader = new BufferedReader(new FileReader(FileManager.toStandardPath(savePath) + "RAE-" + dictionaryWords + "words.txt"));
            String line;
            String[] tokens;
            int count = 0;
            while ((line = bufferedReader.readLine()) != null && count < dictionaryWords) {
                tokens = line.split(",");
                word2 = filterPipe.singleFilter(tokens[0]);
                if (tokens[0].equalsIgnoreCase(word2) && tokens[0].length() > 1) {
                    generalDictionarty.put(tokens[0], Double.parseDouble(tokens[1]));
                    count++;
                }
            }
            bufferedReader.close();
        } catch (IOException ex) {
            Logger.getLogger(ConceptWeirdness.class.getName()).log(Level.SEVERE, null, ex);
        }
        //Specializaed Dictionary
        Corpus corpus = CorpusIO.loadCorpus("conceptAllTerms.dat");
        TObjectIntMap<String> wordList = corpus.getWordCount();
        int totalWords = 0;
        for (String word : wordList.keySet()) {
            totalWords += wordList.get(word);
        }
        for (String word : wordList.keySet()) {
            word2 = filterPipe.singleFilter(word);
            if (word.equalsIgnoreCase(word2) && word.length() > 1) {
                specializedDictionarty.put(word, (double) wordList.get(word) / (double) totalWords);
            }
        }
    }

    public void weirdness() {
        Map<String, Double> weirdness = new HashMap<String, Double>();
        for (String word : specializedDictionarty.keySet()) {
            if (generalDictionarty.containsKey(word)) {
                weirdness.put(word, specializedDictionarty.get(word) / generalDictionarty.get(word));
            } else {
                weirdness.put(word, specializedDictionarty.get(word) / ((double) 1 / (double) totalRAE));
            }
        }
        BufferedWriter bufferedWriter = null;
        try {
            String savePath = Configuration.EXPERIMENTS_DIR + File.separator + ConceptWeirdness.class.getSimpleName();
            FileManager.verifyAndCreatePath(savePath);
            bufferedWriter = new BufferedWriter(new FileWriter(new File(FileManager.toStandardPath(savePath) + Configuration.KNOWLEDGE_SOURCE_NAME + "-" + "ConceptWeirdness" + startDate + ".csv")));
            for (String word : weirdness.keySet()) {
                bufferedWriter.write(word + "," + weirdness.get(word) + "\n");
            }
            bufferedWriter.flush();
            bufferedWriter.close();
        } catch (IOException ex) {
            Logger.getLogger(ConceptWeirdness.class.getName()).log(Level.SEVERE, null, ex);
        } finally {
            try {
                bufferedWriter.close();
            } catch (IOException ex) {
                Logger.getLogger(ConceptWeirdness.class.getName()).log(Level.SEVERE, null, ex);
            }
        }

    }

    public void test() {
        System.out.println("General sample");
        int c = 0;
        for (String word : generalDictionarty.keySet()) {
            System.out.println(word + "->" + generalDictionarty.get(word));
            c++;
            if (c > 1000) {
                break;
            }
        }
        System.out.println("Specialized sample");
        c = 0;
        for (String word : specializedDictionarty.keySet()) {
            System.out.println(word + "->" + specializedDictionarty.get(word));
            c++;
            if (c > 1000) {
                break;
            }
        }
    }

    public static void main(String[] args) throws Exception {
        ConceptWeirdness conceptWeirdness = new ConceptWeirdness("/media/ldatos/tesis/CREA_total2.TXT");
        conceptWeirdness.conceptAllTermsCorpus();
        conceptWeirdness.filterRAEDictionary();
        conceptWeirdness.loadDictionaries();
        conceptWeirdness.weirdness();
        //        conceptWeirdness.test();
    }
}
